#
# Text Mining Example
# http://www.rdatamining.com/examples/text-mining
#
library(tm)

load_data <- function(filename) {
  # Read file and construct corpus
  lines <- scan(filename, character(0), sep='\n')
  corpus <- Corpus(VectorSource(lines))
  
  # Cleanup/prep
  corpus <- tm_map(corpus, tolower)
  corpus <- tm_map(corpus, removePunctuation)
  stop_words <- c(stopwords('english'))
  corpus <- tm_map(corpus, removeWords, stop_words )
  
  corpus
}

terms <- function(corpus) {
  # tm_map(corpus, stemDocument)
  tdm <- TermDocumentMatrix(corpus, control=list(minWordLength=1))
  inspect(tdm[266:270,31:40])
  findFreqTers(tdm)
  findFreqTerms(tdm)
  findFreqTerms(tdm, lowfreq=10)
  findAssocs(tdm, 'sox')
  findAssocs(tdm, 'sox', 0.30)
  
  m <- as.matrix(dtm)
  sort(rowSums(m), decreasing=TRUE)
  
  

}

print_middle_freqs <- function(tdm) {
  ft <- findFreqTerms(tdm, lowfreq=10, highfreq=30)
  for(i in 1:length(ft)) {
    print(ft[i])
    print(findAssocs(tdm, ft[i], 0.3))
  }
}

doclust <- function(tdm) {
  z <- hclust(dist(tdm))  
  plot(z)
}


#*****************************************************************************#
#corpus <- load_data('../bbkb/data/mturk/wiki.answers.com.txt')
# nead read table corpus <- load_data('../bbkb/data/mturk/Batch_644396_batch_results.csv')
tmp <- cbind(lines, inspect(corpus))

